import scrapy
import re
import json
import time
from mySpider.items import JingDongComment


class JingDongSpider(scrapy.Spider):
    name = "JingDongSpider"
    heads = {
        ":authority": "sclub.jd.com",
        ":method": "GET",
        ":path": "/comment/productPageComments.action?callback=fetchJSON_comment98vv1&productId=100002672294&score=0&sortType=5&page=0&pageSize=10&isShadowSku=100001490211&fold=1",
        ":scheme": "https",
        "accept": "*/*",
        "accept-encoding": "gzip, deflate, br",
        "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
        "referer": "https://item.jd.com/100002672294.html",
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
    }

    url = "https://sclub.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98vv1&productId=100002672294&score=0&sortType=5&page=%s&pageSize=10&isShadowSku=100001490211&fold=1"

    def start_requests(self):

        for page in range(5):
            url = self.url % page

            # 手动发送一个请求
            yield scrapy.Request(url=url, callback=self.parse, headers=self.heads)

    # 回调函数
    def parse(self, response):
        '''
        :param response: 响应体
        :return:
        '''
        print("回调函数")
        p = re.compile(r'[(](.*)[)]', re.S)  # 贪婪匹配
        r = re.findall(p, response.text)

        jsonstr = json.loads(r[0])

        comments = jsonstr["comments"]

        for comment in comments:
            content = comment["content"]
            id = comment["id"]
            score = comment["score"]
            nickname = comment["nickname"]
            userLevelName = comment["userLevelName"]
            userClientShow = comment["userClientShow"]
            referenceName = comment["referenceName"]

            # 构建自定义类对象
            item = JingDongComment()
            item["content"] = content
            item["id"] = id
            item["score"] = score
            item["nickname"] = nickname
            item["userLevelName"] = userLevelName
            item["userClientShow"] = userClientShow
            item["referenceName"] = referenceName

            # 将自定义类的对象发送给piplines
            yield item

        time.sleep(1000)
